package crawler;

/**
 * @description:
 * @authoer: Max Woods
 * @create:2018/7/2 11:28
 **/

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.parser.ParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileOutputStream;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Pattern;

/**
 *@auther: Max Woods
 *@create:2018/7/2 11:28
 **/
public class CrawlerShouXi  extends WebCrawler {

    private static final Logger logger = LoggerFactory.getLogger(CrawlerPage.class);

    final Pattern fileFilterPattern = Pattern.compile(".*\\.(zip|mp4|exe|css|gif|js|jpg)(\\?)*.*$",Pattern.CASE_INSENSITIVE);
    final Pattern NoDataPagePattern = Pattern.compile(".*(shouxi.com/user/login|shouxi.com/news/).*",Pattern.CASE_INSENSITIVE);
    final Pattern DataPagePattern = Pattern.compile(".*((.jsp)?\\?G=.*|(.jsp)?\\?T=.*\\&G=.*|(.jsp)?\\?P=.*\\&G=.*|(.jsp)?\\?OBY=.*\\&G=.*)",Pattern.CASE_INSENSITIVE);
    final static Pattern goodsPagePattern = Pattern.compile(".*(/item.*\\?id=).*$",Pattern.CASE_INSENSITIVE);

    private FileOutputStream fileOut;

    final HSSFWorkbook workbook = new HSSFWorkbook();

    final HSSFSheet sheet = workbook.createSheet();

    final Set<String> urlSet=new HashSet<>();

    static int rownum=0;

    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();
        boolean machted =href.contains("shouxi.com/")&&!href.startsWith("http://bbs");
        machted=machted&&!fileFilterPattern.matcher(href).matches();
        machted=machted&&!NoDataPagePattern.matcher(href).matches();
        machted=machted&&!goodsPagePattern.matcher(href).matches();
        if(machted)
        {
            logger.info("=========匹配到url:{}",href);
        }
        else
        {
            logger.info("=========丢弃url:{}",href);
        }
        return machted;
    }

    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        ParseData parseData = page.getParseData();
        if (!(parseData instanceof HtmlParseData)) return;
        logger.info("分析页面链接：{}", url);
        Set<WebURL> webURLS = parseData.getOutgoingUrls();
        for (WebURL webURL : webURLS) {
            String link = webURL.getURL();
            if (goodsPagePattern.matcher(link).matches()) {
                if(urlSet.add(link)) {
                    logger.info("{}数据提取链接:{}", rownum++, link);
                }
            }
        }
    }

    @Override
    public void onStart() {
        SimpleDateFormat sdf=new SimpleDateFormat("yyyyMMddHHmmss");
        try {
            String filePath="d:/首席" + sdf.format(new Date()) + ".xls";
            fileOut = new FileOutputStream(filePath);
            logger.info("创建数据文件：{}",filePath);
        }
        catch (Exception e)
        {
        }
    }

    @Override
    public void onBeforeExit() {
        try {
            rownum=0;
            for(String url:urlSet)
            {
                HSSFRow row = sheet.createRow(rownum++);
                HSSFCell cell = row.createCell(0);
                cell.setCellValue(url);
            }
            workbook.write(fileOut);
            workbook.close();
            fileOut.close();
        }
        catch (Exception e)
        {
        }
    }
}
